package main

import (
	"flag"
	"fmt"
	"os"
	"bufio"
	"github.com/golang/glog"
	"code.google.com/p/fetchbolt/db"
)

func main() {
	var (
		host = flag.String("host", "",
		        "scheme://host[:port]")
		dbpath = flag.String("dbpath", "/tmp/fetchbolt.db",
			"path to store link database")
		dumpLinkDb = flag.Bool("dumplinkdb", false,
			"instead of crawling, dump the link database to stdout as nid,name,list-of-child-nids")
		dumpLinkPairs = flag.Bool("dumplinkpairs", false,
			"instead of crawling, dump the link database to stdout as parent,child pairs")
		dumpQueue = flag.Bool("dumpqueue", false,
			"instead of crawling, dump the db fetch queue to stdout")
		obeyRobots = flag.Bool("obeyrobots", true,
			"obey robots.txt, except crawl delay if -maxcrawldelay given")
		crawlDelay = flag.Duration("crawldelay", 0,
			"if non-zero delay gives default crawl delay (robots.txt can override it)")
	)
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, "Usage: ", os.Args[0], ` [options] -host url-without-path

Indexes specified host and records its contents in the file given by dbpath.

Options:
`)
		flag.PrintDefaults()
	}
	flag.Parse()
	defer glog.Flush()

	if *dumpLinkDb || *dumpLinkPairs || *dumpQueue {
		db, err := db.OpenDbForUrl(*dbpath, *host)
		if err != nil {
			glog.Fatalf("Error opening db: %v", err)
		}
		stdout := bufio.NewWriter(os.Stdout)

		if *dumpLinkDb {
			err = db.WriteEdgesCsv(stdout)
		} else if *dumpLinkPairs {
			err = db.WriteEdgePairsCsv(stdout)
		} else if *dumpQueue {
			err = db.WriteQueueCsv(stdout)
		}
		if err == nil {
			err = stdout.Flush()
		}
		if err != nil {
			glog.Fatalf("Error dumping db queue as csv: %v", err)
		}
		return
	}

	d, err := newDispatcher(*dbpath, *host, *obeyRobots, *crawlDelay)
	if err != nil {
		glog.Fatalf("Error creating dispatcher: %v", err)
	}
	d.main()
}

